package dyyx;

import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.cn.smart.SmartChineseAnalyzer;
import org.apache.lucene.analysis.standard.StandardAnalyzer;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;

public class AnalyzerTest {
	
	public static void main(String[] args) throws Exception {
		
		System.out.println("AnalyzerTest");
		
		
		String msg = MsgUtil.getMsg();
		// using the default stopword list
		SmartChineseAnalyzer smartChineseAnalyzer = new SmartChineseAnalyzer();
		
	    Analyzer standardAnalyzer = new StandardAnalyzer();

        String fieldName = "test";
        TokenStream tokenStream = smartChineseAnalyzer.tokenStream(fieldName, msg);
	    
        tokenStream.reset();
	    while(tokenStream.incrementToken()){
	    	 CharTermAttribute charTermAttribute = tokenStream.getAttribute(CharTermAttribute.class); 
	    	 System.out.println(charTermAttribute.toString());
	    }
	    System.out.println("==============");
        tokenStream = standardAnalyzer.tokenStream(fieldName, msg);
	    
        tokenStream.reset();
	    while(tokenStream.incrementToken()){
	    	 CharTermAttribute charTermAttribute = tokenStream.getAttribute(CharTermAttribute.class); 
	    	 System.out.println(charTermAttribute.toString());
	    }
	
	    System.out.println("==============");
	    
	    
        smartChineseAnalyzer = new SmartChineseAnalyzer();
		
	    standardAnalyzer = new StandardAnalyzer();

	    msg = MsgUtil.getMsgEn();
	    
	    //  java.lang.IllegalStateException: TokenStream contract violation: close() call missing
	    
        tokenStream = smartChineseAnalyzer.tokenStream(fieldName, msg);
	    
        tokenStream.reset();
	    while(tokenStream.incrementToken()){
	    	 CharTermAttribute charTermAttribute = tokenStream.getAttribute(CharTermAttribute.class); 
	    	 System.out.println(charTermAttribute.toString());
	    }
	    System.out.println("==============");
        tokenStream = standardAnalyzer.tokenStream(fieldName, msg);
	    
        tokenStream.reset();
	    while(tokenStream.incrementToken()){
	    	 CharTermAttribute charTermAttribute = tokenStream.getAttribute(CharTermAttribute.class); 
	    	 System.out.println(charTermAttribute.toString());
	    }
	
	    System.out.println("==============");
	    
	    
	}
}
